import os,csv,sys
from datetime import datetime
import time,re

# Array of dicts which contain the distinct values for each attribute as keys
distinctVal=[]
# column header of the attribute
colNames=[]
#colAlphaNum contains 'a' if the attribute is alphanumeric and 'n' if it is numeric
colAlphaNum=['n']*273


baseDir='/root/hearst'
os.chdir(baseDir)
newModel=open('newModeling.csv')
validation=open('validation.csv')
distinctValFile=open('distinctVal.txt','w')
csvfile=csv.reader(newModel,delimiter=',',quotechar='"')
validationfile=csv.reader(validation,delimiter=',',quotechar='"')

#function to convert string class in to numerical class label
def classToNum(cls):
    if cls=='Y':
        return '+1'
    else:
        return '-1'
    return '-1'

#function to split categorical attribute header to multiple nominal attribute headers
def cat2NomHeaders(col,colDict,attribId=None):
   if col in ['NUMBER_OF_CHLDRN_18_OR_LESS','LENGTH_OF_RESIDENCE']:
       return [col+'^num',col+'^',col+'^U']
   elif re.match(r'^(DMR|BUYER)_.+',col):
       return [col+'^num',col+'^',col+'^Y',col+'^U']
   elif attribId==ageStartIdx:
        return ['AGE^0-10','AGE^11-20','AGE^21-30','AGE^31-40','AGE^41-60','AGE^61+']
   elif attribId==genStartIdx:
        return ['GENDER^B','GENDER^F','GENDER^M','GENDER^U']
   elif attribId==HHStatIdx:
        return ['HHSTAT^H','HHSTAT^P','HHSTAT^U','HHSTAT^W','HHSTAT^Y']
   elif attribId==traitStartIdx:
        trait=[]
        for key in sorted(traitMap.keys()):
            trait.append(traitMap[key][0])
        trait.append('TRAIT_OTHERS')
        return trait
   else:
       tup=['']*len(colDict)
       for ctr,key in enumerate(colDict.keys()):
          tup[colDict[key]-1]=(col+'^'+key)
       return tup

#function to categorical value to nominal values
def cat2NomVal(key,colDict,attribId=None):
    if attribId==None:
        tup=[0]*len(colDict)
        idx=colDict[key]
        tup[idx-1]=1
        return tup
    
    else:
        if re.match(r'^(DMR|BUYER)_.+',colNames[attribId]) or colNames[attribId] in ['LENGTH_OF_RESIDENCE','NUMBER_OF_CHLDRN_18_OR_LESS']:
            try:
                if colNames[attribId]=='LENGTH_OF_RESIDENCE':
                    return [normalize(float(key),0,61),0,0] # format is [number,'','U']
                elif colNames[attribId]=='NUMBER_OF_CHLDRN_18_OR_LESS':
                    return [normalize(float(key),0,7),0,0] #  format is [number,'','U']
                else:
                    return [normalize(float(key),0,9),0,0,0] #  format is [number,'','Y','U']
            except ValueError:
                if key=='':
                    return [0,1,0,0]
                elif key=='Y':
                    return [0,0,1,0]
                elif key=='U' and (colNames[attribId]=='LENGTH_OF_RESIDENCE' or colNames[attribId]=='NUMBER_OF_CHLDRN_18_OR_LESS'):
                    return [0,0,1]
                elif key=='U':
                    return [0,0,0,1]

#function to find the distinct values,datatype for each attribute
def findDistinctVal(dataFile):
    for row in dataFile:
        if len(row)>273:
            print row
            sys.exit(2)
        for ctr,val in enumerate(row):
            if ctr==newidIdx:
                distinctVal[ctr][val+','+row[mailingidIdx]]=1
            elif ctr>= traitStartIdx and ctr <=traitStartIdx+127:
                continue
            elif ctr==272:
                distinctVal[ctr][str(int(time.mktime(datetime.strptime(val,"%a, %m/%d/%y %H:%M %p").date().timetuple())))]=1
            else:
                distinctVal[ctr][val]=1
            #find if the col is numeric or alphanumeric
            try:
                if colAlphaNum[ctr]=='n' and val!='':
                    tmpvar=float(val)
            except ValueError:
               colAlphaNum[ctr]='a'

def findMinMax():
    minMaxMap={}
    for loopctr,dtype in enumerate(colAlphaNum[2:len(colAlphaNum)+1]):
        ctr=loopctr+2
        minval,maxval=(0,0)
        if ctr==newidIdx:
            continue
        if ctr==cityIdx:
            minval=min(distCenUSA.values())
            maxval=max(distCenUSA.values())            
        elif dtype=='n':
            val=[]
            for i in distinctVal[ctr].keys():
                if i!='':
                    val.append(float(i))
            if len(val)==0:
                val.append(0)
            minval=min(val)
            maxval=max(val)
        minMaxMap[ctr]=(minval,maxval)
    return minMaxMap

def normalize(val,min,max):
    if max==min and min==0:
        return 0
    elif max==min:
        return 1
    else:
        return (val-min)/(max-min)
        

def getDistCenUSA():
  distMap={}  
  distFile=open('distFromCentreOfUSA_new.csv')
  for cityDist in distFile:
      cityDist=cityDist.rstrip('\n')
      cityDistSplit=cityDist.split('^')
      dist=float(cityDistSplit[1])
      distMap[cityDistSplit[0]]=float(dist)
  return distMap

def getTraitMap():
  traitMap={}  
  traitFile=open('traits.csv')
  for trait in traitFile:
      traitInfo=trait.rstrip('\n')
      traitInfo=trait.split(',')
      traitMap[(traitInfo[1],traitInfo[2])]=(traitInfo[0],0)
      for ctr,key in enumerate(sorted(traitMap.keys())):
          traitMap[key]=(traitMap[key][0],ctr)
  return traitMap

def getTraitDataFromRow(traitStart,traitMap,row):
    trait_cd=row[traitStart:traitStart+64]
    trait_typ_cd=row[traitStart+64:traitStart+128]       
    traitData=[0]*(len(traitMap.keys())+1)
    for ctr in xrange(0,len(trait_cd)):
        try:
           traitData[traitMap[(trait_cd[ctr],trait_typ_cd[ctr])][1]]=1
        except KeyError:
            traitData[len(traitData)-1]=1
    s=len(traitData)
    v=len(traitMap)
    return traitData

def getAgeDataFromRow(ageStart,row):
    ageData=[0]*6 #[0-10,11-20,21-30,31-40,41-60,61+]
    for age in row[ageStart:ageStart+8]:
        if age=='':
            continue
        elif int(age)>0 and int(age)<=10:
            ageData[0]+=1
        elif int(age)>10 and int(age)<=20:
            ageData[1]+=1
        elif int(age)>20 and int(age)<=30:
            ageData[2]+=1
        elif int(age)>30 and int(age)<=40:
            ageData[3]+=1
        elif int(age)>40 and int(age)<=60:
            ageData[4]+=1
        elif int(age)>60:
            ageData[5]+=1
    return ageData

def getGenderDataFromRow(genStart,row):
    genderData=[0]*4 #[B,F,M,U]
    for gender in row[genStart:genStart+8]:
        if gender=='':
            continue
        elif (gender)=='B':
            genderData[0]+=1
        elif (gender)=='F':
            genderData[1]+=1
        elif (gender)=='M':
            genderData[2]+=1
        elif (gender)=='U':
            genderData[3]+=1
    return genderData

def getHHStatDataFromRow(HHstart,row):
    HHStatData=[0]*5 #[H,P,U,W,Y]
    for HHStat in row[HHstart:HHstart+8]:
        if HHStat=='':
            continue
        elif (HHStat)=='H':
            HHStatData[0]+=1
        elif (HHStat)=='P':
            HHStatData[1]+=1
        elif (HHStat)=='U':
            HHStatData[2]+=1
        elif (HHStat)=='W':
            HHStatData[3]+=1
        elif (HHStat)=='Y':
            HHStatData[4]+=1
    return HHStatData



def getHeader(row):
    header=[]
    for loopctr,item in enumerate(row[2:len(row)+1]):
        ctr=loopctr+2
        if ctr==newidIdx:
            continue
        elif ctr>cityIdx and ctr<=cityIdx+2:
            continue
        elif ctr==cityIdx:
            header.append('CITY')
        elif ctr==mailingidIdx or colNames[ctr] in ['new_list_id','list_new_brand_cd','list_new_ed_cd','LENGTH_OF_RESIDENCE','NUMBER_OF_CHLDRN_18_OR_LESS'] or re.match(r'^(DMR|BUYER)_.+',colNames[ctr]):
            header+=cat2NomHeaders(colNames[ctr],distinctVal[ctr])
        elif ctr==ageStartIdx:
            header+=cat2NomHeaders(colNames[ctr],distinctVal[ctr],ctr)
        elif ctr> ageStartIdx and ctr <=ageStartIdx+7:
            continue
        elif ctr==genStartIdx:
            header+=cat2NomHeaders(colNames[ctr],distinctVal[ctr],ctr)
        elif ctr> genStartIdx and ctr <=genStartIdx+7:
            continue
        elif ctr==HHStatIdx:
            header+=cat2NomHeaders(colNames[ctr],distinctVal[ctr],ctr)
        elif ctr> HHStatIdx and ctr <=HHStatIdx+7:
            continue
        elif ctr==traitStartIdx:
            header+=cat2NomHeaders(colNames[ctr],distinctVal[ctr],ctr)
        elif ctr> traitStartIdx and ctr <=traitStartIdx+127:
            continue
        elif ctr==272:
            header.append('Mailing_date')
        elif colAlphaNum[ctr]=='n':
            header.append(colNames[ctr])
        elif colAlphaNum[ctr]=='a' and len(distinctVal[ctr])<1000:
            header+=cat2NomHeaders(colNames[ctr],distinctVal[ctr],ctr)
    return header
            
def getDataFromRow(row):
    data=[]
    uid=[]
    for loopctr,item in enumerate(row[2:len(row)+1]):
        ctr=loopctr+2
        if ctr==newidIdx:
            uid.append(row[newidIdx]+','+row[mailingidIdx])
        elif ctr>cityIdx and ctr<=cityIdx+2:
            continue
        elif ctr==cityIdx:
            try:
               data.append(normalize(distCenUSA[item+','+row[stateIdx]],minMax[ctr][0],minMax[ctr][1]))
            except KeyError:
                 data.append(0.99)
                 print 'KeyError for City '+item+','+row[stateIdx]
            #add the logic for getting the distance of the city from the geo center of entire U.S.A
        #convert campaign id in to a categorical attribute
        elif ctr==mailingidIdx or colNames[ctr] in ['new_list_id','list_new_brand_cd','list_new_ed_cd']:
            data+=cat2NomVal(item,distinctVal[ctr])
        elif re.match(r'^(DMR|BUYER)_.+',colNames[ctr]) or colNames[ctr] in ['LENGTH_OF_RESIDENCE','NUMBER_OF_CHLDRN_18_OR_LESS']:
            data+=cat2NomVal(item,distinctVal[ctr],ctr)
        elif ctr==ageStartIdx:
            data+=getAgeDataFromRow(ctr,row)
        elif ctr> ageStartIdx and ctr <=ageStartIdx+7:
            continue
        elif ctr==genStartIdx:
            data+=getGenderDataFromRow(ctr,row)
        elif ctr> genStartIdx and ctr <=genStartIdx+7:
            continue
        elif ctr==HHStatIdx:
            data+=getHHStatDataFromRow(ctr,row)
        elif ctr> HHStatIdx and ctr <=HHStatIdx+7:
            continue
        elif ctr==traitStartIdx:
            data+=getTraitDataFromRow(traitStartIdx,traitMap,row)
        elif ctr> traitStartIdx and ctr <=traitStartIdx+127:
            continue
        elif ctr==272:
            data.append(normalize(time.mktime(datetime.strptime(item,"%a, %m/%d/%y %H:%M %p").date().timetuple()),minMax[ctr][0],minMax[ctr][1]))
        elif colAlphaNum[ctr]=='n':
            if item=='':
                data.append('')
            else:
                data.append(normalize(float(item),minMax[ctr][0],minMax[ctr][1]))
        elif colAlphaNum[ctr]=='a' and len(distinctVal[ctr])<1000:
            try:
               data+=cat2NomVal(item,distinctVal[ctr])
            except KeyError:
               distinctValFile.write('the column is '+colNames[ctr]+'\n')
               distinctValFile.write('the Key is '+item+'\n')
               distinctValFile.close()
               sys.exit(-1)
        
        
    sparseMap={}
    for ctr,val in enumerate(data):
        if val!='' and val!='0' and val!=0:
           sparseMap[ctr+1]=str(round(val,6))
    return sparseMap,uid


#Start of the program
#get the program start time
tstart=datetime.now()
#populate the distMap
#Dict which contains the distance in kms between center of the entire USA and each city
distCenUSA=getDistCenUSA()
traitMap=getTraitMap()

#populate the column headers
for colname in csvfile.next():
    colNames.append(colname)
    distinctVal.append({})

#get the index for the columns which need special treatment
newidIdx=colNames.index('new_id')
mailingidIdx=colNames.index('new_mailing_id')
cityIdx=colNames.index('CITY')
stateIdx=cityIdx+2
traitStartIdx=colNames.index('trait_cd1')
ageStartIdx=colNames.index('age_hh_mem1')
genStartIdx=colNames.index('gender_hh_mem1')
HHStatIdx=colNames.index('statcd_hh_mem1')
colNames[newidIdx]=colNames[newidIdx]+','+colNames[mailingidIdx]

#find the distinct values from Modeling data file
findDistinctVal(csvfile);

#ignore the first line containing headers in the validation file
validationfile.next()

#find the distinct values from Validation file
findDistinctVal(validationfile);

#setting the datefield as numeric, as date is converted in to unix timestamp
colAlphaNum[272]='n'

for ctr,colDict in enumerate(distinctVal):
    for ctr,item in enumerate(sorted(colDict.keys())):
           colDict[item]=ctr+1

#Dict {attr_key:(min,max)} which contains the tuple (min,max) for each attribute, will be used for Normalization
minMax=findMinMax()

#write the distinct values for attributes with distinct value count < 1000
for ctr,name in enumerate(colNames):
    distinctValFile.write("#column:"+name+","+colAlphaNum[ctr]+","+str(len(distinctVal[ctr]))+"\n")
    if len(distinctVal[ctr])<1000:
       for item in distinctVal[ctr].keys():
          distinctValFile.write(item+'\n')

## the distinct values for each attribute is now available in memory
## if the number of distinct values is less than 1000 we convert each distinct value in to an attribute i.e nominal to binary
## if the number of distinct values is greater than 1000, we assign an index to each distinct value and convert the nominal attribute in to numerical attribute
## remove the attribute new_id


#rest the file seek position for Training and Validation data to zero
newModel.seek(0)
validation.seek(0)

# create 2 seperate training files one for open_flg and one for click_flg
open_flgTrain=open('open_flag.train','w')
click_flgTrain=open('click_flag.train','w')

#ingore header which is first line in the file
csvfile.next()
validationfile.next()

#write the new header
newHeader=getHeader(colNames)
distinctValFile.write("The new header is"+'\n')
for ctr,column in enumerate(newHeader):
    distinctValFile.write(str(ctr+1)+':'+column+'\n')
distinctValFile.close()

#preparing the training files for svm light
for row in csvfile:
    modelSparseMap,uid=getDataFromRow(row)

    open_flgTrain.write(classToNum(row[1])+' ')
    for key in sorted(modelSparseMap.keys()):
        open_flgTrain.write(str(key)+':'+modelSparseMap[key]+' ')
    open_flgTrain.write('\n')

    if classToNum(row[1])=='+1':    
        click_flgTrain.write(classToNum(row[0])+' ')
        for key in sorted(modelSparseMap.keys()):
            click_flgTrain.write(str(key)+':'+modelSparseMap[key]+' ')
        click_flgTrain.write('\n')

open_flgTrain.close()
click_flgTrain.close()    

#preparing the test files for svm light from validation file
    
valid_test=open('valid.test','w')
valid_id=open('valid_id.test','w')

for row in validationfile:
    validSparseMap,uid=getDataFromRow(row)
    valid_test.write('0 ')
    for key in sorted(validSparseMap.keys()):
        valid_test.write(str(key)+':'+validSparseMap[key]+' ')
    valid_test.write('\n')
    valid_id.write(uid[0]+'\n')

    
valid_test.close()
valid_id.close()

tend=datetime.now()
c=tend-tstart
print str(c.seconds/60)+' Minutes to complete'
